package com.wsc.indexer;



import java.io.UnsupportedEncodingException;
import java.security.NoSuchAlgorithmException;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.LimitTokenCountAnalyzer;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;

import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.Index;

import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;

import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;

import org.apache.lucene.util.Version;

import com.wsc.crawler.init.CrawlerConfig;
import com.wsc.indexer.utils.IndexerURLBean;
import com.wsc.indexer.utils.MD5CheckSum;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;

public class LuceneIndexer {

    /*
     * static block which executes automatically when obj is reffered.
     */
   private  File indexDir;
   private  Directory fsDir;
   private  Analyzer anlyzer;
   private  Analyzer ltcAn;
   private  IndexWriterConfig iwConf;
   private  IndexWriter indexWriter ;
   
   /*
    * this constructor takes a dir name in which it creates index or indexes data to already existing index.
    */
   public LuceneIndexer(String dir){
        try {
        	
        	
        	// Modified this line
        	
            CrawlerConfig conf=new CrawlerConfig();
            
            indexDir=new File(dir);
            fsDir = FSDirectory.open(indexDir);

             anlyzer = new StandardAnalyzer(Version.LUCENE_36);
             ltcAn = new LimitTokenCountAnalyzer(anlyzer,Integer.MAX_VALUE);

            iwConf = new IndexWriterConfig(Version.LUCENE_36,ltcAn);
            iwConf.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);

            indexWriter = new IndexWriter(fsDir,iwConf);
        } catch (IOException ex) {
            Logger.getLogger(LuceneIndexer.class.getName()).log(Level.SEVERE, null, ex);
        }
        }
        
        
   // To Index a list of URLS
   
        public void index(ArrayList urls) throws IOException{
                 
        	IndexerURLBean bean = null;
            Iterator itr=urls.iterator();

            // Get first object from the arrayList which contains meta informaton about,
            // Which crawler crawled the sent urls, 
            if(itr.hasNext())
             bean=(IndexerURLBean) itr.next();
            
            // Get next objects which are instances of url class.
       while(itr.hasNext()){
            try {
                /*
                 * Here assign url with the urls in a webpage
                 */               
                String url =(String)itr.next();
                
                // Get unique Hash for each unique URL
                
                String md5 =MD5CheckSum.MD5(url);
                
                System.out.println("[INFO]:{URL Indexed in "+this.getClass().getName()+"}["+url+"] its hash ["+md5+"]");
                Document d = new Document();
                d.add(new Field("url",url,Store.YES,Index.NOT_ANALYZED));
                d.add(new Field("md5",md5,Store.YES,Index.NOT_ANALYZED));
                d.add(new Field("date",(new Date()).toString(),Store.YES,Index.NO));
                
                // get the crawledby(crawler name/IP) field from the IndexerURLBean
                
                d.add(new Field("crawledby",bean.getCrawledBy(),Store.YES,Index.NOT_ANALYZED));
                
                indexWriter.addDocument(d);
                
            } catch (NoSuchAlgorithmException ex) {
                Logger.getLogger(LuceneIndexer.class.getName()).log(Level.SEVERE, null, ex);
            } catch (UnsupportedEncodingException ex) {
                Logger.getLogger(LuceneIndexer.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
        int numDocs = indexWriter.numDocs();

        indexWriter.forceMerge(1);
        indexWriter.commit();
        
        System.out.println("[INFO]:{Index Directory in "+this.getClass().getName()+"} [" + indexDir.getCanonicalPath()+"]");
        System.out.println("[INFO]:{num docs in Index}["+ numDocs+"]");
    }
        
        
        // To Index Single URL
        public void index(IndexerURLBean bean){
        try {
                
                Document d = new Document();
                
                // Get to be indexed URL from bean.
                
                String stringurl=bean.getUrl();
                stringurl=stringurl.trim(); // removes leading and tailing spaces in url.
                String md5=MD5CheckSum.MD5(stringurl);
                d.add(new Field("url",stringurl,Store.YES,Index.NOT_ANALYZED));
                d.add(new Field("md5",md5, Store.YES,Index.NOT_ANALYZED));
                d.add(new Field("date",(new Date()).toString(),Store.YES,Index.NO));
                
                // get the crawledby(crawler name/IP) field from the IndexerURLBean
                
                d.add(new Field("crawledby",bean.getCrawledBy(),Store.YES,Index.NOT_ANALYZED));
                
                indexWriter.addDocument(d);
                 int numDocs = indexWriter.numDocs();
                indexWriter.forceMerge(1);
                indexWriter.commit();
                
                System.out.println("[INFO]:{Num os docs are}["+numDocs+"]");
        } catch (CorruptIndexException ex) {
            Logger.getLogger(LuceneIndexer.class.getName()).log(Level.SEVERE, null, ex);
        } catch (IOException ex) {
            Logger.getLogger(LuceneIndexer.class.getName()).log(Level.SEVERE, null, ex);
        } catch (NoSuchAlgorithmException ex) {
            Logger.getLogger(LuceneIndexer.class.getName()).log(Level.SEVERE, null, ex);
        } 
        }
        
        
        /*
         * close after all.
         */
        public void closeIndex(){
        try {
        	
        	// IndexWriter is not null, then close it.
        	
        	if(this.indexWriter != null)
            indexWriter.close();
        } catch (CorruptIndexException ex) {
            Logger.getLogger(LuceneIndexer.class.getName()).log(Level.SEVERE, null, ex);
        } catch (IOException ex) {
            Logger.getLogger(LuceneIndexer.class.getName()).log(Level.SEVERE, null, ex);
        }
        }
        
        public int getNumberOfUrlsInIndex(){
        try {
            return this.indexWriter.numDocs();
        } catch (IOException ex) {
            Logger.getLogger(LuceneIndexer.class.getName()).log(Level.SEVERE, null, ex);
        } 
        return 0;
        }
    

}